{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from browser import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_exps = [\n",
    "  'C100_DenseAdam',\n",
    "  'C100_DenseSGD',\n",
    "  'C100_SparseAdam',\n",
    "  'C100_SparseSGD',\n",
    "  'C10_DenseAdam',\n",
    "  'C10_DenseSGD',\n",
    "  'C10_SparseAdam',\n",
    "  'C10_SparseSGD',\n",
    "]\n",
    "\n",
    "old_exps = [\n",
    "  'VGG19DenseTest9v2',\n",
    "  'VGG19SparseFull',\n",
    "  'VGG19SparseFull-short',\n",
    "  'VGG19SparseTest9b2',\n",
    "]\n",
    "\n",
    "exps = new_exps + old_exps\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No experiment state found for experiment /Users/lsouza/nta/results/VGG19SparseTest9b2\n"
     ]
    }
   ],
   "source": [
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(616, 62)"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>epoch_test_accuracy</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>epoch_noise_accuracy</th>\n",
       "      <th>mean_accuracy</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>epoch_mean_accuracy</th>\n",
       "      <th>...</th>\n",
       "      <th>test_batch_size</th>\n",
       "      <th>test_batches_in_epoch</th>\n",
       "      <th>upload_dir</th>\n",
       "      <th>use_max_pooling</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>weight_sparsity</th>\n",
       "      <th>stop</th>\n",
       "      <th>learning_rate_gamma</th>\n",
       "      <th>lr_step_schedule</th>\n",
       "      <th>momentum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_learning_rate=0.001,weight_decay=0.003543</td>\n",
       "      <td>0.3050</td>\n",
       "      <td>0.3147</td>\n",
       "      <td>153</td>\n",
       "      <td>0.0645</td>\n",
       "      <td>0.0892</td>\n",
       "      <td>54</td>\n",
       "      <td>0.18475</td>\n",
       "      <td>0.19485</td>\n",
       "      <td>145</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.003543</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_learning_rate=0.001,weight_decay=0.0068785</td>\n",
       "      <td>0.2972</td>\n",
       "      <td>0.3189</td>\n",
       "      <td>162</td>\n",
       "      <td>0.0650</td>\n",
       "      <td>0.0887</td>\n",
       "      <td>162</td>\n",
       "      <td>0.18110</td>\n",
       "      <td>0.20380</td>\n",
       "      <td>162</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.006879</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_learning_rate=0.0003,weight_decay=0.0054007</td>\n",
       "      <td>0.6342</td>\n",
       "      <td>0.6443</td>\n",
       "      <td>159</td>\n",
       "      <td>0.1416</td>\n",
       "      <td>0.1879</td>\n",
       "      <td>156</td>\n",
       "      <td>0.38790</td>\n",
       "      <td>0.40795</td>\n",
       "      <td>162</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.005401</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_learning_rate=0.003,weight_decay=0.0007664</td>\n",
       "      <td>0.2215</td>\n",
       "      <td>0.2394</td>\n",
       "      <td>144</td>\n",
       "      <td>0.0490</td>\n",
       "      <td>0.0671</td>\n",
       "      <td>63</td>\n",
       "      <td>0.13525</td>\n",
       "      <td>0.14775</td>\n",
       "      <td>144</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.000766</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_learning_rate=0.03,weight_decay=0.0012639</td>\n",
       "      <td>0.0344</td>\n",
       "      <td>0.0502</td>\n",
       "      <td>136</td>\n",
       "      <td>0.0099</td>\n",
       "      <td>0.0418</td>\n",
       "      <td>33</td>\n",
       "      <td>0.02215</td>\n",
       "      <td>0.04210</td>\n",
       "      <td>33</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.001264</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 62 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Experiment Name  test_accuracy  \\\n",
       "0    0_learning_rate=0.001,weight_decay=0.003543         0.3050   \n",
       "1   1_learning_rate=0.001,weight_decay=0.0068785         0.2972   \n",
       "2  2_learning_rate=0.0003,weight_decay=0.0054007         0.6342   \n",
       "3   3_learning_rate=0.003,weight_decay=0.0007664         0.2215   \n",
       "4    4_learning_rate=0.03,weight_decay=0.0012639         0.0344   \n",
       "\n",
       "   test_accuracy_max  epoch_test_accuracy  noise_accuracy  noise_accuracy_max  \\\n",
       "0             0.3147                  153          0.0645              0.0892   \n",
       "1             0.3189                  162          0.0650              0.0887   \n",
       "2             0.6443                  159          0.1416              0.1879   \n",
       "3             0.2394                  144          0.0490              0.0671   \n",
       "4             0.0502                  136          0.0099              0.0418   \n",
       "\n",
       "   epoch_noise_accuracy  mean_accuracy  mean_accuracy_max  \\\n",
       "0                    54        0.18475            0.19485   \n",
       "1                   162        0.18110            0.20380   \n",
       "2                   156        0.38790            0.40795   \n",
       "3                    63        0.13525            0.14775   \n",
       "4                    33        0.02215            0.04210   \n",
       "\n",
       "   epoch_mean_accuracy  ...  test_batch_size  test_batches_in_epoch  \\\n",
       "0                  145  ...              128                    500   \n",
       "1                  162  ...              128                    500   \n",
       "2                  162  ...              128                    500   \n",
       "3                  144  ...              128                    500   \n",
       "4                   33  ...              128                    500   \n",
       "\n",
       "                upload_dir  use_max_pooling weight_decay  weight_sparsity  \\\n",
       "0  s3://lsouza/ray/results             True     0.003543              1.0   \n",
       "1  s3://lsouza/ray/results             True     0.006879              1.0   \n",
       "2  s3://lsouza/ray/results             True     0.005401              1.0   \n",
       "3  s3://lsouza/ray/results             True     0.000766              1.0   \n",
       "4  s3://lsouza/ray/results             True     0.001264              1.0   \n",
       "\n",
       "   stop  learning_rate_gamma  lr_step_schedule  momentum  \n",
       "0   NaN                  NaN               NaN       NaN  \n",
       "1   NaN                  NaN               NaN       NaN  \n",
       "2   NaN                  NaN               NaN       NaN  \n",
       "3   NaN                  NaN               NaN       NaN  \n",
       "4   NaN                  NaN               NaN       NaN  \n",
       "\n",
       "[5 rows x 62 columns]"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Experiment Name           149_batches_in_epoch=575,boost_strength=1.0613...\n",
       "test_accuracy                                                        0.1924\n",
       "test_accuracy_max                                                    0.1924\n",
       "epoch_test_accuracy                                                      19\n",
       "noise_accuracy                                                       0.1458\n",
       "noise_accuracy_max                                                   0.1469\n",
       "epoch_noise_accuracy                                                     17\n",
       "mean_accuracy                                                        0.1691\n",
       "mean_accuracy_max                                                    0.1691\n",
       "epoch_mean_accuracy                                                      19\n",
       "epochs                                                                   20\n",
       "start_learning_rate                                               0.0973613\n",
       "end_learning_rate                                                 0.0973613\n",
       "early_stop                                                                0\n",
       "experiment_file_name      /Users/lsouza/nta/results/VGG19SparseFull-shor...\n",
       "trial_time                                                          59.0119\n",
       "mean_epoch_time                                                     2.95059\n",
       "trial_train_time                                                    50.6388\n",
       "mean_epoch_train_time                                               2.53194\n",
       "batch_size                                                              128\n",
       "batches_in_epoch                                                        575\n",
       "batches_in_first_epoch                                                  600\n",
       "block_sizes                                                             3.2\n",
       "boost_strength                                                      1.06132\n",
       "boost_strength_factor                                              0.799318\n",
       "checkpoint_at_end                                                      True\n",
       "cnn_kernel_size                                                           3\n",
       "cnn_out_channels                                                      294.4\n",
       "cnn_percent_on                                                     0.174313\n",
       "cnn_weight_sparsity                                                0.773573\n",
       "                                                ...                        \n",
       "dataset                                                            CIFAR100\n",
       "experiment                                                             grid\n",
       "first_epoch_batch_size                                                    4\n",
       "gpu_percentage                                                         0.14\n",
       "input_shape                                                         22.3333\n",
       "iterations                                                              164\n",
       "k_inference_factor                                                  1.12695\n",
       "learning_rate                                                     0.0973613\n",
       "linear_n                                                                NaN\n",
       "linear_percent_on                                                       NaN\n",
       "name                                                        VGG19SparseFull\n",
       "network_type                                                            vgg\n",
       "num_cpus                                                                 31\n",
       "num_gpus                                                                  4\n",
       "optimizer                                                               NaN\n",
       "output_size                                                             100\n",
       "path                                                          ~/nta/results\n",
       "repetitions                                                             150\n",
       "restore_supported                                                      True\n",
       "sync_function             aws s3 sync `dirname {local_dir}` {remote_dir}...\n",
       "test_batch_size                                                         128\n",
       "test_batches_in_epoch                                                   500\n",
       "upload_dir                                          s3://lsouza/ray/results\n",
       "use_max_pooling                                                        True\n",
       "weight_decay                                                    0.000384426\n",
       "weight_sparsity                                                         NaN\n",
       "stop                                                                    NaN\n",
       "learning_rate_gamma                                                0.177053\n",
       "lr_step_schedule                                                      101.5\n",
       "momentum                                                           0.644845\n",
       "Name: 615, Length: 62, dtype: object"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filters = (df['name'].str.startswith('C100_SparseSGD'))\n",
    "         \n",
    "(df[filters]\n",
    "  [['start_learning_rate', 'end_learning_rate', 'early_stop']]\n",
    "  .head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_learning_rate</th>\n",
       "      <th>end_learning_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>0.0010</td>\n",
       "      <td>1.000000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>0.0100</td>\n",
       "      <td>1.000000e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0.0030</td>\n",
       "      <td>3.000000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>1.000000e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>0.0030</td>\n",
       "      <td>3.000000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>1.000000e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>4.733613e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>0.0300</td>\n",
       "      <td>1.440304e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>3.342565e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>4.296152e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>0.0010</td>\n",
       "      <td>1.000000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>0.0010</td>\n",
       "      <td>1.000000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>1.114699e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>3.310691e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.0030</td>\n",
       "      <td>2.121552e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>3.505160e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>1.286253e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>8.182201e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>1.645728e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>1.567386e-02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    start_learning_rate  end_learning_rate\n",
       "35               0.0010       1.000000e-03\n",
       "36               0.0100       1.000000e-02\n",
       "37               0.0030       3.000000e-03\n",
       "38               0.0001       1.000000e-04\n",
       "39               0.0030       3.000000e-03\n",
       "40               0.1000       1.000000e-01\n",
       "41               0.0003       4.733613e-05\n",
       "42               0.0300       1.440304e-03\n",
       "43               0.1000       3.342565e-06\n",
       "44               0.0003       4.296152e-05\n",
       "45               0.0010       1.000000e-03\n",
       "46               0.0010       1.000000e-03\n",
       "47               0.1000       1.114699e-02\n",
       "48               0.0001       3.310691e-06\n",
       "49               0.0030       2.121552e-03\n",
       "50               0.0003       3.505160e-07\n",
       "51               0.1000       1.286253e-03\n",
       "52               0.0003       8.182201e-05\n",
       "53               0.0003       1.645728e-04\n",
       "54               0.1000       1.567386e-02"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filters = (df['name'].str.startswith('C100_SparseAdam'))\n",
    "         \n",
    "(df[filters]\n",
    "  [['start_learning_rate', 'end_learning_rate']]\n",
    "  .head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14993308514938847"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['start_learning_rate'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.84348080181885e-09"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['end_learning_rate'].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.12012987012987013"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df['early_stop']) / len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>trial_time</th>\n",
       "      <th>batches_in_epoch</th>\n",
       "      <th>epochs</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">CIFAR10</th>\n",
       "      <th>C10_DenseAdam</th>\n",
       "      <td>0.92</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.56</td>\n",
       "      <td>507.36</td>\n",
       "      <td>500</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C10_DenseSGD</th>\n",
       "      <td>0.93</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.58</td>\n",
       "      <td>500.30</td>\n",
       "      <td>500</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C10_SparseAdam</th>\n",
       "      <td>0.92</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.61</td>\n",
       "      <td>569.74</td>\n",
       "      <td>500</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C10_SparseSGD</th>\n",
       "      <td>0.92</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.67</td>\n",
       "      <td>558.73</td>\n",
       "      <td>500</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">CIFAR100</th>\n",
       "      <th>C100_DenseAdam</th>\n",
       "      <td>0.64</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.17</td>\n",
       "      <td>509.73</td>\n",
       "      <td>500</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_DenseSGD</th>\n",
       "      <td>0.71</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.23</td>\n",
       "      <td>502.12</td>\n",
       "      <td>500</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseAdam</th>\n",
       "      <td>0.68</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.22</td>\n",
       "      <td>652.49</td>\n",
       "      <td>500</td>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseSGD</th>\n",
       "      <td>0.67</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.32</td>\n",
       "      <td>562.81</td>\n",
       "      <td>500</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VGG19DenseTest9v2</th>\n",
       "      <td>0.72</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.22</td>\n",
       "      <td>348.41</td>\n",
       "      <td>400</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VGG19SparseFull</th>\n",
       "      <td>0.71</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.31</td>\n",
       "      <td>560.62</td>\n",
       "      <td>599</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            test_accuracy_max  test_accuracy  \\\n",
       "dataset  name                                                  \n",
       "CIFAR10  C10_DenseAdam                   0.92           0.92   \n",
       "         C10_DenseSGD                    0.93           0.93   \n",
       "         C10_SparseAdam                  0.92           0.92   \n",
       "         C10_SparseSGD                   0.92           0.92   \n",
       "CIFAR100 C100_DenseAdam                  0.64           0.63   \n",
       "         C100_DenseSGD                   0.71           0.71   \n",
       "         C100_SparseAdam                 0.68           0.68   \n",
       "         C100_SparseSGD                  0.67           0.67   \n",
       "         VGG19DenseTest9v2               0.72           0.72   \n",
       "         VGG19SparseFull                 0.71           0.70   \n",
       "\n",
       "                            noise_accuracy_max  noise_accuracy  trial_time  \\\n",
       "dataset  name                                                                \n",
       "CIFAR10  C10_DenseAdam                    0.60            0.56      507.36   \n",
       "         C10_DenseSGD                     0.65            0.58      500.30   \n",
       "         C10_SparseAdam                   0.63            0.61      569.74   \n",
       "         C10_SparseSGD                    0.69            0.67      558.73   \n",
       "CIFAR100 C100_DenseAdam                   0.24            0.17      509.73   \n",
       "         C100_DenseSGD                    0.27            0.23      502.12   \n",
       "         C100_SparseAdam                  0.26            0.22      652.49   \n",
       "         C100_SparseSGD                   0.33            0.32      562.81   \n",
       "         VGG19DenseTest9v2                0.24            0.22      348.41   \n",
       "         VGG19SparseFull                  0.33            0.31      560.62   \n",
       "\n",
       "                            batches_in_epoch  epochs  \n",
       "dataset  name                                         \n",
       "CIFAR10  C10_DenseAdam                   500     164  \n",
       "         C10_DenseSGD                    500     164  \n",
       "         C10_SparseAdam                  500     164  \n",
       "         C10_SparseSGD                   500     164  \n",
       "CIFAR100 C100_DenseAdam                  500     164  \n",
       "         C100_DenseSGD                   500     164  \n",
       "         C100_SparseAdam                 500     192  \n",
       "         C100_SparseSGD                  500     164  \n",
       "         VGG19DenseTest9v2               400     200  \n",
       "         VGG19SparseFull                 599     164  "
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.groupby(['dataset', 'name'])['test_accuracy_max', 'test_accuracy', 'noise_accuracy_max', \n",
    "                                 'noise_accuracy', 'trial_time', 'batches_in_epoch',\n",
    "                                 'epochs']\n",
    "                                 .max().round(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>trial_time</th>\n",
       "      <th>batches_in_epoch</th>\n",
       "      <th>epochs</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">CIFAR10</th>\n",
       "      <th>C10_DenseAdam</th>\n",
       "      <td>0.69</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.38</td>\n",
       "      <td>269.53</td>\n",
       "      <td>500.00</td>\n",
       "      <td>89.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C10_DenseSGD</th>\n",
       "      <td>0.92</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.53</td>\n",
       "      <td>282.71</td>\n",
       "      <td>500.00</td>\n",
       "      <td>93.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C10_SparseAdam</th>\n",
       "      <td>0.71</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.41</td>\n",
       "      <td>247.46</td>\n",
       "      <td>500.00</td>\n",
       "      <td>71.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C10_SparseSGD</th>\n",
       "      <td>0.87</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.58</td>\n",
       "      <td>264.54</td>\n",
       "      <td>500.00</td>\n",
       "      <td>79.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">CIFAR100</th>\n",
       "      <th>C100_DenseAdam</th>\n",
       "      <td>0.24</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.06</td>\n",
       "      <td>270.49</td>\n",
       "      <td>500.00</td>\n",
       "      <td>87.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_DenseSGD</th>\n",
       "      <td>0.66</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.20</td>\n",
       "      <td>216.69</td>\n",
       "      <td>500.00</td>\n",
       "      <td>71.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseAdam</th>\n",
       "      <td>0.38</td>\n",
       "      <td>0.37</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.10</td>\n",
       "      <td>195.42</td>\n",
       "      <td>500.00</td>\n",
       "      <td>58.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseSGD</th>\n",
       "      <td>0.48</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.18</td>\n",
       "      <td>155.17</td>\n",
       "      <td>500.00</td>\n",
       "      <td>47.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VGG19DenseTest9v2</th>\n",
       "      <td>0.39</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.13</td>\n",
       "      <td>158.71</td>\n",
       "      <td>400.00</td>\n",
       "      <td>91.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VGG19SparseFull</th>\n",
       "      <td>0.30</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.14</td>\n",
       "      <td>116.88</td>\n",
       "      <td>455.47</td>\n",
       "      <td>39.10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            test_accuracy_max  test_accuracy  \\\n",
       "dataset  name                                                  \n",
       "CIFAR10  C10_DenseAdam                   0.69           0.69   \n",
       "         C10_DenseSGD                    0.92           0.90   \n",
       "         C10_SparseAdam                  0.71           0.70   \n",
       "         C10_SparseSGD                   0.87           0.85   \n",
       "CIFAR100 C100_DenseAdam                  0.24           0.22   \n",
       "         C100_DenseSGD                   0.66           0.65   \n",
       "         C100_SparseAdam                 0.38           0.37   \n",
       "         C100_SparseSGD                  0.48           0.46   \n",
       "         VGG19DenseTest9v2               0.39           0.39   \n",
       "         VGG19SparseFull                 0.30           0.29   \n",
       "\n",
       "                            noise_accuracy_max  noise_accuracy  trial_time  \\\n",
       "dataset  name                                                                \n",
       "CIFAR10  C10_DenseAdam                    0.42            0.38      269.53   \n",
       "         C10_DenseSGD                     0.61            0.53      282.71   \n",
       "         C10_SparseAdam                   0.46            0.41      247.46   \n",
       "         C10_SparseSGD                    0.63            0.58      264.54   \n",
       "CIFAR100 C100_DenseAdam                   0.08            0.06      270.49   \n",
       "         C100_DenseSGD                    0.23            0.20      216.69   \n",
       "         C100_SparseAdam                  0.12            0.10      195.42   \n",
       "         C100_SparseSGD                   0.22            0.18      155.17   \n",
       "         VGG19DenseTest9v2                0.15            0.13      158.71   \n",
       "         VGG19SparseFull                  0.16            0.14      116.88   \n",
       "\n",
       "                            batches_in_epoch  epochs  \n",
       "dataset  name                                         \n",
       "CIFAR10  C10_DenseAdam                500.00   89.56  \n",
       "         C10_DenseSGD                 500.00   93.56  \n",
       "         C10_SparseAdam               500.00   71.81  \n",
       "         C10_SparseSGD                500.00   79.65  \n",
       "CIFAR100 C100_DenseAdam               500.00   87.61  \n",
       "         C100_DenseSGD                500.00   71.71  \n",
       "         C100_SparseAdam              500.00   58.58  \n",
       "         C100_SparseSGD               500.00   47.19  \n",
       "         VGG19DenseTest9v2            400.00   91.25  \n",
       "         VGG19SparseFull              455.47   39.10  "
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.groupby(['dataset', 'name'])['test_accuracy_max', 'test_accuracy', 'noise_accuracy_max', \n",
    "                                 'noise_accuracy', 'trial_time', 'batches_in_epoch',\n",
    "                                 'epochs']\n",
    "                                 .mean().round(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>noise_accuracy</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">CIFAR10</th>\n",
       "      <th>C10_DenseAdam</th>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C10_DenseSGD</th>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C10_SparseAdam</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C10_SparseSGD</th>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">CIFAR100</th>\n",
       "      <th>C100_DenseAdam</th>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_DenseSGD</th>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseAdam</th>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseSGD</th>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VGG19DenseTest9v2</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VGG19SparseFull</th>\n",
       "      <td>450</td>\n",
       "      <td>450</td>\n",
       "      <td>450</td>\n",
       "      <td>450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            test_accuracy_max  test_accuracy  \\\n",
       "dataset  name                                                  \n",
       "CIFAR10  C10_DenseAdam                      9              9   \n",
       "         C10_DenseSGD                       9              9   \n",
       "         C10_SparseAdam                    16             16   \n",
       "         C10_SparseSGD                     17             17   \n",
       "CIFAR100 C100_DenseAdam                    18             18   \n",
       "         C100_DenseSGD                     17             17   \n",
       "         C100_SparseAdam                   36             36   \n",
       "         C100_SparseSGD                    36             36   \n",
       "         VGG19DenseTest9v2                  8              8   \n",
       "         VGG19SparseFull                  450            450   \n",
       "\n",
       "                            noise_accuracy_max  noise_accuracy  \n",
       "dataset  name                                                   \n",
       "CIFAR10  C10_DenseAdam                       9               9  \n",
       "         C10_DenseSGD                        9               9  \n",
       "         C10_SparseAdam                     16              16  \n",
       "         C10_SparseSGD                      17              17  \n",
       "CIFAR100 C100_DenseAdam                     18              18  \n",
       "         C100_DenseSGD                      17              17  \n",
       "         C100_SparseAdam                    36              36  \n",
       "         C100_SparseSGD                     36              36  \n",
       "         VGG19DenseTest9v2                   8               8  \n",
       "         VGG19SparseFull                   450             450  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.groupby(['dataset', 'name'])['test_accuracy_max', 'test_accuracy', 'noise_accuracy_max', \n",
    "                                 'noise_accuracy', 'experiment_time', 'batches_in_epoch']\n",
    "                                 .mean().round(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How is the custom early stopping behaving?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>epochs</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>noise_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>C10_DenseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.6972</td>\n",
       "      <td>0.6972</td>\n",
       "      <td>0.3899</td>\n",
       "      <td>0.3715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>C10_DenseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.9043</td>\n",
       "      <td>0.9020</td>\n",
       "      <td>0.5527</td>\n",
       "      <td>0.4671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>C10_DenseAdam</td>\n",
       "      <td>113</td>\n",
       "      <td>0.8505</td>\n",
       "      <td>0.8490</td>\n",
       "      <td>0.4874</td>\n",
       "      <td>0.4012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>C10_DenseAdam</td>\n",
       "      <td>108</td>\n",
       "      <td>0.9217</td>\n",
       "      <td>0.9217</td>\n",
       "      <td>0.5980</td>\n",
       "      <td>0.5604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>C10_DenseAdam</td>\n",
       "      <td>42</td>\n",
       "      <td>0.9160</td>\n",
       "      <td>0.9156</td>\n",
       "      <td>0.5555</td>\n",
       "      <td>0.4918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>C10_DenseAdam</td>\n",
       "      <td>133</td>\n",
       "      <td>0.7525</td>\n",
       "      <td>0.7516</td>\n",
       "      <td>0.4061</td>\n",
       "      <td>0.3638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>C10_DenseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2093</td>\n",
       "      <td>0.1859</td>\n",
       "      <td>0.2086</td>\n",
       "      <td>0.2026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>C10_DenseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>C10_DenseAdam</td>\n",
       "      <td>42</td>\n",
       "      <td>0.8927</td>\n",
       "      <td>0.8927</td>\n",
       "      <td>0.5138</td>\n",
       "      <td>0.4440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>C10_DenseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.9280</td>\n",
       "      <td>0.9263</td>\n",
       "      <td>0.6358</td>\n",
       "      <td>0.5562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>C10_DenseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.9270</td>\n",
       "      <td>0.9264</td>\n",
       "      <td>0.5805</td>\n",
       "      <td>0.5536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>C10_DenseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.9249</td>\n",
       "      <td>0.9240</td>\n",
       "      <td>0.5977</td>\n",
       "      <td>0.5497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>C10_DenseSGD</td>\n",
       "      <td>111</td>\n",
       "      <td>0.9292</td>\n",
       "      <td>0.9279</td>\n",
       "      <td>0.5619</td>\n",
       "      <td>0.5411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>C10_DenseSGD</td>\n",
       "      <td>59</td>\n",
       "      <td>0.9174</td>\n",
       "      <td>0.9097</td>\n",
       "      <td>0.6022</td>\n",
       "      <td>0.5361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>C10_DenseSGD</td>\n",
       "      <td>56</td>\n",
       "      <td>0.9204</td>\n",
       "      <td>0.9178</td>\n",
       "      <td>0.6467</td>\n",
       "      <td>0.5218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>C10_DenseSGD</td>\n",
       "      <td>58</td>\n",
       "      <td>0.9204</td>\n",
       "      <td>0.9191</td>\n",
       "      <td>0.6197</td>\n",
       "      <td>0.5754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>C10_DenseSGD</td>\n",
       "      <td>46</td>\n",
       "      <td>0.8929</td>\n",
       "      <td>0.8725</td>\n",
       "      <td>0.6098</td>\n",
       "      <td>0.4424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>C10_DenseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8828</td>\n",
       "      <td>0.8072</td>\n",
       "      <td>0.6135</td>\n",
       "      <td>0.4722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.7793</td>\n",
       "      <td>0.7778</td>\n",
       "      <td>0.4911</td>\n",
       "      <td>0.4604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.7706</td>\n",
       "      <td>0.7687</td>\n",
       "      <td>0.4219</td>\n",
       "      <td>0.3921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.4724</td>\n",
       "      <td>0.4584</td>\n",
       "      <td>0.3146</td>\n",
       "      <td>0.3010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>96</td>\n",
       "      <td>0.8687</td>\n",
       "      <td>0.8646</td>\n",
       "      <td>0.5619</td>\n",
       "      <td>0.4432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>82</td>\n",
       "      <td>0.8505</td>\n",
       "      <td>0.8496</td>\n",
       "      <td>0.5005</td>\n",
       "      <td>0.4763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>79</td>\n",
       "      <td>0.9089</td>\n",
       "      <td>0.9075</td>\n",
       "      <td>0.5824</td>\n",
       "      <td>0.4917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>76</td>\n",
       "      <td>0.7626</td>\n",
       "      <td>0.7626</td>\n",
       "      <td>0.4898</td>\n",
       "      <td>0.3976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2468</td>\n",
       "      <td>0.1847</td>\n",
       "      <td>0.2086</td>\n",
       "      <td>0.1728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>52</td>\n",
       "      <td>0.8827</td>\n",
       "      <td>0.8776</td>\n",
       "      <td>0.5514</td>\n",
       "      <td>0.4102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.3139</td>\n",
       "      <td>0.2661</td>\n",
       "      <td>0.2289</td>\n",
       "      <td>0.1920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>42</td>\n",
       "      <td>0.9095</td>\n",
       "      <td>0.9080</td>\n",
       "      <td>0.6034</td>\n",
       "      <td>0.5401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>44</td>\n",
       "      <td>0.9008</td>\n",
       "      <td>0.9008</td>\n",
       "      <td>0.6170</td>\n",
       "      <td>0.5996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>59</td>\n",
       "      <td>0.9156</td>\n",
       "      <td>0.9156</td>\n",
       "      <td>0.6242</td>\n",
       "      <td>0.6104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.5972</td>\n",
       "      <td>0.5972</td>\n",
       "      <td>0.3667</td>\n",
       "      <td>0.2643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>47</td>\n",
       "      <td>0.9044</td>\n",
       "      <td>0.9030</td>\n",
       "      <td>0.6338</td>\n",
       "      <td>0.6122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>C10_SparseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2306</td>\n",
       "      <td>0.2044</td>\n",
       "      <td>0.2229</td>\n",
       "      <td>0.2229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.8992</td>\n",
       "      <td>0.8973</td>\n",
       "      <td>0.6070</td>\n",
       "      <td>0.5997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.9043</td>\n",
       "      <td>0.9033</td>\n",
       "      <td>0.6415</td>\n",
       "      <td>0.6159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.8361</td>\n",
       "      <td>0.8330</td>\n",
       "      <td>0.6768</td>\n",
       "      <td>0.6687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.8571</td>\n",
       "      <td>0.8542</td>\n",
       "      <td>0.6573</td>\n",
       "      <td>0.6383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>97</td>\n",
       "      <td>0.9233</td>\n",
       "      <td>0.9223</td>\n",
       "      <td>0.6668</td>\n",
       "      <td>0.6298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>81</td>\n",
       "      <td>0.8975</td>\n",
       "      <td>0.8942</td>\n",
       "      <td>0.6444</td>\n",
       "      <td>0.6184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>81</td>\n",
       "      <td>0.9101</td>\n",
       "      <td>0.9059</td>\n",
       "      <td>0.6523</td>\n",
       "      <td>0.6291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>48</td>\n",
       "      <td>0.9151</td>\n",
       "      <td>0.9144</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>0.5807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>63</td>\n",
       "      <td>0.9092</td>\n",
       "      <td>0.9078</td>\n",
       "      <td>0.6887</td>\n",
       "      <td>0.6578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>67</td>\n",
       "      <td>0.9104</td>\n",
       "      <td>0.9100</td>\n",
       "      <td>0.6721</td>\n",
       "      <td>0.6437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7633</td>\n",
       "      <td>0.6576</td>\n",
       "      <td>0.5416</td>\n",
       "      <td>0.4555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>49</td>\n",
       "      <td>0.8888</td>\n",
       "      <td>0.8874</td>\n",
       "      <td>0.6823</td>\n",
       "      <td>0.6554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>74</td>\n",
       "      <td>0.9113</td>\n",
       "      <td>0.9099</td>\n",
       "      <td>0.6413</td>\n",
       "      <td>0.5882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7365</td>\n",
       "      <td>0.7167</td>\n",
       "      <td>0.5925</td>\n",
       "      <td>0.5150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>58</td>\n",
       "      <td>0.8595</td>\n",
       "      <td>0.8532</td>\n",
       "      <td>0.6858</td>\n",
       "      <td>0.6616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7937</td>\n",
       "      <td>0.6986</td>\n",
       "      <td>0.5731</td>\n",
       "      <td>0.2337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>C10_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.7990</td>\n",
       "      <td>0.7990</td>\n",
       "      <td>0.4635</td>\n",
       "      <td>0.3973</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               name  epochs  test_accuracy_max  test_accuracy  \\\n",
       "107   C10_DenseAdam     164             0.6972         0.6972   \n",
       "108   C10_DenseAdam     164             0.9043         0.9020   \n",
       "109   C10_DenseAdam     113             0.8505         0.8490   \n",
       "110   C10_DenseAdam     108             0.9217         0.9217   \n",
       "111   C10_DenseAdam      42             0.9160         0.9156   \n",
       "112   C10_DenseAdam     133             0.7525         0.7516   \n",
       "113   C10_DenseAdam      20             0.2093         0.1859   \n",
       "114   C10_DenseAdam      20             0.1000         0.1000   \n",
       "115   C10_DenseAdam      42             0.8927         0.8927   \n",
       "116    C10_DenseSGD     164             0.9280         0.9263   \n",
       "117    C10_DenseSGD     164             0.9270         0.9264   \n",
       "118    C10_DenseSGD     164             0.9249         0.9240   \n",
       "119    C10_DenseSGD     111             0.9292         0.9279   \n",
       "120    C10_DenseSGD      59             0.9174         0.9097   \n",
       "121    C10_DenseSGD      56             0.9204         0.9178   \n",
       "122    C10_DenseSGD      58             0.9204         0.9191   \n",
       "123    C10_DenseSGD      46             0.8929         0.8725   \n",
       "124    C10_DenseSGD      20             0.8828         0.8072   \n",
       "125  C10_SparseAdam     164             0.7793         0.7778   \n",
       "126  C10_SparseAdam     164             0.7706         0.7687   \n",
       "127  C10_SparseAdam     164             0.4724         0.4584   \n",
       "128  C10_SparseAdam      96             0.8687         0.8646   \n",
       "129  C10_SparseAdam      82             0.8505         0.8496   \n",
       "130  C10_SparseAdam      79             0.9089         0.9075   \n",
       "131  C10_SparseAdam      76             0.7626         0.7626   \n",
       "132  C10_SparseAdam      20             0.2468         0.1847   \n",
       "133  C10_SparseAdam      52             0.8827         0.8776   \n",
       "134  C10_SparseAdam      20             0.3139         0.2661   \n",
       "135  C10_SparseAdam      42             0.9095         0.9080   \n",
       "136  C10_SparseAdam      44             0.9008         0.9008   \n",
       "137  C10_SparseAdam      59             0.9156         0.9156   \n",
       "138  C10_SparseAdam      20             0.5972         0.5972   \n",
       "139  C10_SparseAdam      47             0.9044         0.9030   \n",
       "140  C10_SparseAdam      20             0.2306         0.2044   \n",
       "141   C10_SparseSGD     164             0.8992         0.8973   \n",
       "142   C10_SparseSGD     164             0.9043         0.9033   \n",
       "143   C10_SparseSGD     164             0.8361         0.8330   \n",
       "144   C10_SparseSGD     164             0.8571         0.8542   \n",
       "145   C10_SparseSGD      97             0.9233         0.9223   \n",
       "146   C10_SparseSGD      81             0.8975         0.8942   \n",
       "147   C10_SparseSGD      81             0.9101         0.9059   \n",
       "148   C10_SparseSGD      48             0.9151         0.9144   \n",
       "149   C10_SparseSGD      63             0.9092         0.9078   \n",
       "150   C10_SparseSGD      67             0.9104         0.9100   \n",
       "151   C10_SparseSGD      20             0.7633         0.6576   \n",
       "152   C10_SparseSGD      49             0.8888         0.8874   \n",
       "153   C10_SparseSGD      74             0.9113         0.9099   \n",
       "154   C10_SparseSGD      20             0.7365         0.7167   \n",
       "155   C10_SparseSGD      58             0.8595         0.8532   \n",
       "156   C10_SparseSGD      20             0.7937         0.6986   \n",
       "157   C10_SparseSGD      20             0.7990         0.7990   \n",
       "\n",
       "     noise_accuracy_max  noise_accuracy  \n",
       "107              0.3899          0.3715  \n",
       "108              0.5527          0.4671  \n",
       "109              0.4874          0.4012  \n",
       "110              0.5980          0.5604  \n",
       "111              0.5555          0.4918  \n",
       "112              0.4061          0.3638  \n",
       "113              0.2086          0.2026  \n",
       "114              0.1000          0.1000  \n",
       "115              0.5138          0.4440  \n",
       "116              0.6358          0.5562  \n",
       "117              0.5805          0.5536  \n",
       "118              0.5977          0.5497  \n",
       "119              0.5619          0.5411  \n",
       "120              0.6022          0.5361  \n",
       "121              0.6467          0.5218  \n",
       "122              0.6197          0.5754  \n",
       "123              0.6098          0.4424  \n",
       "124              0.6135          0.4722  \n",
       "125              0.4911          0.4604  \n",
       "126              0.4219          0.3921  \n",
       "127              0.3146          0.3010  \n",
       "128              0.5619          0.4432  \n",
       "129              0.5005          0.4763  \n",
       "130              0.5824          0.4917  \n",
       "131              0.4898          0.3976  \n",
       "132              0.2086          0.1728  \n",
       "133              0.5514          0.4102  \n",
       "134              0.2289          0.1920  \n",
       "135              0.6034          0.5401  \n",
       "136              0.6170          0.5996  \n",
       "137              0.6242          0.6104  \n",
       "138              0.3667          0.2643  \n",
       "139              0.6338          0.6122  \n",
       "140              0.2229          0.2229  \n",
       "141              0.6070          0.5997  \n",
       "142              0.6415          0.6159  \n",
       "143              0.6768          0.6687  \n",
       "144              0.6573          0.6383  \n",
       "145              0.6668          0.6298  \n",
       "146              0.6444          0.6184  \n",
       "147              0.6523          0.6291  \n",
       "148              0.6553          0.5807  \n",
       "149              0.6887          0.6578  \n",
       "150              0.6721          0.6437  \n",
       "151              0.5416          0.4555  \n",
       "152              0.6823          0.6554  \n",
       "153              0.6413          0.5882  \n",
       "154              0.5925          0.5150  \n",
       "155              0.6858          0.6616  \n",
       "156              0.5731          0.2337  \n",
       "157              0.4635          0.3973  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = ['epochs', 'test_accuracy_max', 'test_accuracy', 'noise_accuracy_max', 'noise_accuracy']\n",
    "df[df['name'].str.startswith('C10_')][['name'] + metrics]\n",
    "\n",
    "# (['dataset', 'name'])['test_accuracy_max', 'test_accuracy', 'noise_accuracy_max', 'noise_accuracy']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>epochs</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>early_stop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>51</td>\n",
       "      <td>0.7115</td>\n",
       "      <td>0.7124</td>\n",
       "      <td>0.2102</td>\n",
       "      <td>0.2676</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>42</td>\n",
       "      <td>0.7124</td>\n",
       "      <td>0.7124</td>\n",
       "      <td>0.2231</td>\n",
       "      <td>0.2433</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>42</td>\n",
       "      <td>0.7055</td>\n",
       "      <td>0.7057</td>\n",
       "      <td>0.1957</td>\n",
       "      <td>0.2495</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>91</td>\n",
       "      <td>0.7007</td>\n",
       "      <td>0.7024</td>\n",
       "      <td>0.2234</td>\n",
       "      <td>0.2434</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>43</td>\n",
       "      <td>0.6990</td>\n",
       "      <td>0.6996</td>\n",
       "      <td>0.2086</td>\n",
       "      <td>0.2336</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>42</td>\n",
       "      <td>0.6936</td>\n",
       "      <td>0.6980</td>\n",
       "      <td>0.1976</td>\n",
       "      <td>0.2203</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>42</td>\n",
       "      <td>0.6928</td>\n",
       "      <td>0.6946</td>\n",
       "      <td>0.2310</td>\n",
       "      <td>0.2594</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.6836</td>\n",
       "      <td>0.6875</td>\n",
       "      <td>0.2044</td>\n",
       "      <td>0.2199</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>95</td>\n",
       "      <td>0.6772</td>\n",
       "      <td>0.6798</td>\n",
       "      <td>0.2124</td>\n",
       "      <td>0.2339</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.6746</td>\n",
       "      <td>0.6766</td>\n",
       "      <td>0.2211</td>\n",
       "      <td>0.2313</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>89</td>\n",
       "      <td>0.6697</td>\n",
       "      <td>0.6759</td>\n",
       "      <td>0.2105</td>\n",
       "      <td>0.2442</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>110</td>\n",
       "      <td>0.6559</td>\n",
       "      <td>0.6580</td>\n",
       "      <td>0.2089</td>\n",
       "      <td>0.2188</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.6507</td>\n",
       "      <td>0.6529</td>\n",
       "      <td>0.2011</td>\n",
       "      <td>0.2125</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.5566</td>\n",
       "      <td>0.5881</td>\n",
       "      <td>0.1333</td>\n",
       "      <td>0.1803</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.5627</td>\n",
       "      <td>0.5627</td>\n",
       "      <td>0.1055</td>\n",
       "      <td>0.2030</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.5035</td>\n",
       "      <td>0.5445</td>\n",
       "      <td>0.1713</td>\n",
       "      <td>0.2223</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.5294</td>\n",
       "      <td>0.5309</td>\n",
       "      <td>0.1735</td>\n",
       "      <td>0.2141</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             name  epochs  test_accuracy  test_accuracy_max  noise_accuracy  \\\n",
       "28  C100_DenseSGD      51         0.7115             0.7124          0.2102   \n",
       "31  C100_DenseSGD      42         0.7124             0.7124          0.2231   \n",
       "26  C100_DenseSGD      42         0.7055             0.7057          0.1957   \n",
       "23  C100_DenseSGD      91         0.7007             0.7024          0.2234   \n",
       "27  C100_DenseSGD      43         0.6990             0.6996          0.2086   \n",
       "32  C100_DenseSGD      42         0.6936             0.6980          0.1976   \n",
       "30  C100_DenseSGD      42         0.6928             0.6946          0.2310   \n",
       "19  C100_DenseSGD     164         0.6836             0.6875          0.2044   \n",
       "22  C100_DenseSGD      95         0.6772             0.6798          0.2124   \n",
       "20  C100_DenseSGD     164         0.6746             0.6766          0.2211   \n",
       "24  C100_DenseSGD      89         0.6697             0.6759          0.2105   \n",
       "21  C100_DenseSGD     110         0.6559             0.6580          0.2089   \n",
       "18  C100_DenseSGD     164         0.6507             0.6529          0.2011   \n",
       "25  C100_DenseSGD      20         0.5566             0.5881          0.1333   \n",
       "34  C100_DenseSGD      20         0.5627             0.5627          0.1055   \n",
       "29  C100_DenseSGD      20         0.5035             0.5445          0.1713   \n",
       "33  C100_DenseSGD      20         0.5294             0.5309          0.1735   \n",
       "\n",
       "    noise_accuracy_max  early_stop  \n",
       "28              0.2676           1  \n",
       "31              0.2433           1  \n",
       "26              0.2495           1  \n",
       "23              0.2434           1  \n",
       "27              0.2336           1  \n",
       "32              0.2203           1  \n",
       "30              0.2594           1  \n",
       "19              0.2199           0  \n",
       "22              0.2339           1  \n",
       "20              0.2313           0  \n",
       "24              0.2442           1  \n",
       "21              0.2188           1  \n",
       "18              0.2125           0  \n",
       "25              0.1803           0  \n",
       "34              0.2030           0  \n",
       "29              0.2223           0  \n",
       "33              0.2141           0  "
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = ['epochs', 'test_accuracy', 'test_accuracy_max', 'noise_accuracy', 'noise_accuracy_max', 'early_stop']\n",
    "df[df['name'].str.startswith('C100_DenseSGD')][['name'] + metrics].sort_values(['test_accuracy_max'], ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>epochs</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>early_stop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.6342</td>\n",
       "      <td>0.6443</td>\n",
       "      <td>0.1416</td>\n",
       "      <td>0.1879</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>110</td>\n",
       "      <td>0.6303</td>\n",
       "      <td>0.6388</td>\n",
       "      <td>0.1572</td>\n",
       "      <td>0.1862</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>49</td>\n",
       "      <td>0.6124</td>\n",
       "      <td>0.6124</td>\n",
       "      <td>0.1715</td>\n",
       "      <td>0.2415</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>62</td>\n",
       "      <td>0.5994</td>\n",
       "      <td>0.6033</td>\n",
       "      <td>0.1321</td>\n",
       "      <td>0.1783</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.2972</td>\n",
       "      <td>0.3189</td>\n",
       "      <td>0.0650</td>\n",
       "      <td>0.0887</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.3050</td>\n",
       "      <td>0.3147</td>\n",
       "      <td>0.0645</td>\n",
       "      <td>0.0892</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.2564</td>\n",
       "      <td>0.2908</td>\n",
       "      <td>0.0534</td>\n",
       "      <td>0.0933</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.2215</td>\n",
       "      <td>0.2394</td>\n",
       "      <td>0.0490</td>\n",
       "      <td>0.0671</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.1485</td>\n",
       "      <td>0.1485</td>\n",
       "      <td>0.0333</td>\n",
       "      <td>0.0541</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0876</td>\n",
       "      <td>0.0888</td>\n",
       "      <td>0.0423</td>\n",
       "      <td>0.0538</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0524</td>\n",
       "      <td>0.0795</td>\n",
       "      <td>0.0231</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0235</td>\n",
       "      <td>0.0700</td>\n",
       "      <td>0.0407</td>\n",
       "      <td>0.0461</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>111</td>\n",
       "      <td>0.0274</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.0148</td>\n",
       "      <td>0.0384</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>164</td>\n",
       "      <td>0.0344</td>\n",
       "      <td>0.0502</td>\n",
       "      <td>0.0099</td>\n",
       "      <td>0.0418</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>90</td>\n",
       "      <td>0.0116</td>\n",
       "      <td>0.0438</td>\n",
       "      <td>0.0107</td>\n",
       "      <td>0.0359</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>51</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>0.0139</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>0.0122</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0098</td>\n",
       "      <td>0.0116</td>\n",
       "      <td>0.0102</td>\n",
       "      <td>0.0105</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>C100_DenseAdam</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              name  epochs  test_accuracy  test_accuracy_max  noise_accuracy  \\\n",
       "2   C100_DenseAdam     164         0.6342             0.6443          0.1416   \n",
       "7   C100_DenseAdam     110         0.6303             0.6388          0.1572   \n",
       "9   C100_DenseAdam      49         0.6124             0.6124          0.1715   \n",
       "17  C100_DenseAdam      62         0.5994             0.6033          0.1321   \n",
       "1   C100_DenseAdam     164         0.2972             0.3189          0.0650   \n",
       "0   C100_DenseAdam     164         0.3050             0.3147          0.0645   \n",
       "5   C100_DenseAdam     164         0.2564             0.2908          0.0534   \n",
       "3   C100_DenseAdam     164         0.2215             0.2394          0.0490   \n",
       "12  C100_DenseAdam      20         0.1485             0.1485          0.0333   \n",
       "16  C100_DenseAdam      20         0.0876             0.0888          0.0423   \n",
       "15  C100_DenseAdam      20         0.0524             0.0795          0.0231   \n",
       "13  C100_DenseAdam      20         0.0235             0.0700          0.0407   \n",
       "6   C100_DenseAdam     111         0.0274             0.0608          0.0148   \n",
       "4   C100_DenseAdam     164         0.0344             0.0502          0.0099   \n",
       "8   C100_DenseAdam      90         0.0116             0.0438          0.0107   \n",
       "14  C100_DenseAdam      51         0.0100             0.0139          0.0100   \n",
       "11  C100_DenseAdam      20         0.0098             0.0116          0.0102   \n",
       "10  C100_DenseAdam      20         0.0100             0.0100          0.0100   \n",
       "\n",
       "    noise_accuracy_max  early_stop  \n",
       "2               0.1879           0  \n",
       "7               0.1862           0  \n",
       "9               0.2415           1  \n",
       "17              0.1783           1  \n",
       "1               0.0887           0  \n",
       "0               0.0892           0  \n",
       "5               0.0933           0  \n",
       "3               0.0671           0  \n",
       "12              0.0541           0  \n",
       "16              0.0538           0  \n",
       "15              0.0506           0  \n",
       "13              0.0461           0  \n",
       "6               0.0384           0  \n",
       "4               0.0418           0  \n",
       "8               0.0359           1  \n",
       "14              0.0122           1  \n",
       "11              0.0105           0  \n",
       "10              0.0100           0  "
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = ['epochs', 'test_accuracy', 'test_accuracy_max', 'noise_accuracy', 'noise_accuracy_max', 'early_stop']\n",
    "df[df['name'].str.startswith('C100_DenseAdam')][['name'] + metrics].sort_values(['test_accuracy_max'], ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_diff</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>noise_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>0.6731</td>\n",
       "      <td>0.6722</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0.2968</td>\n",
       "      <td>0.2610</td>\n",
       "      <td>0.0358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>0.6720</td>\n",
       "      <td>0.6720</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2761</td>\n",
       "      <td>0.2524</td>\n",
       "      <td>0.0237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>0.6707</td>\n",
       "      <td>0.6707</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2696</td>\n",
       "      <td>0.2444</td>\n",
       "      <td>0.0252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>0.6706</td>\n",
       "      <td>0.6684</td>\n",
       "      <td>0.0022</td>\n",
       "      <td>0.2524</td>\n",
       "      <td>0.2256</td>\n",
       "      <td>0.0268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>0.6683</td>\n",
       "      <td>0.6683</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2559</td>\n",
       "      <td>0.2484</td>\n",
       "      <td>0.0075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>0.6669</td>\n",
       "      <td>0.6640</td>\n",
       "      <td>0.0029</td>\n",
       "      <td>0.2506</td>\n",
       "      <td>0.2310</td>\n",
       "      <td>0.0196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>0.6667</td>\n",
       "      <td>0.6663</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.2583</td>\n",
       "      <td>0.2400</td>\n",
       "      <td>0.0183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>0.6604</td>\n",
       "      <td>0.6592</td>\n",
       "      <td>0.0012</td>\n",
       "      <td>0.2516</td>\n",
       "      <td>0.2491</td>\n",
       "      <td>0.0025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>0.6566</td>\n",
       "      <td>0.6507</td>\n",
       "      <td>0.0059</td>\n",
       "      <td>0.2889</td>\n",
       "      <td>0.2749</td>\n",
       "      <td>0.0140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0.6545</td>\n",
       "      <td>0.6522</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.2839</td>\n",
       "      <td>0.2682</td>\n",
       "      <td>0.0157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>0.6475</td>\n",
       "      <td>0.6475</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2571</td>\n",
       "      <td>0.2533</td>\n",
       "      <td>0.0038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>0.6461</td>\n",
       "      <td>0.6324</td>\n",
       "      <td>0.0137</td>\n",
       "      <td>0.2725</td>\n",
       "      <td>0.2360</td>\n",
       "      <td>0.0365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>0.6428</td>\n",
       "      <td>0.6411</td>\n",
       "      <td>0.0017</td>\n",
       "      <td>0.2978</td>\n",
       "      <td>0.2749</td>\n",
       "      <td>0.0229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>0.6284</td>\n",
       "      <td>0.6255</td>\n",
       "      <td>0.0029</td>\n",
       "      <td>0.2823</td>\n",
       "      <td>0.2218</td>\n",
       "      <td>0.0605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>0.6124</td>\n",
       "      <td>0.6117</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.2302</td>\n",
       "      <td>0.2156</td>\n",
       "      <td>0.0146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>0.5928</td>\n",
       "      <td>0.5657</td>\n",
       "      <td>0.0271</td>\n",
       "      <td>0.2499</td>\n",
       "      <td>0.2111</td>\n",
       "      <td>0.0388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>0.5913</td>\n",
       "      <td>0.5900</td>\n",
       "      <td>0.0013</td>\n",
       "      <td>0.2576</td>\n",
       "      <td>0.2460</td>\n",
       "      <td>0.0116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.5574</td>\n",
       "      <td>0.5451</td>\n",
       "      <td>0.0123</td>\n",
       "      <td>0.3280</td>\n",
       "      <td>0.3231</td>\n",
       "      <td>0.0049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>0.5424</td>\n",
       "      <td>0.5393</td>\n",
       "      <td>0.0031</td>\n",
       "      <td>0.1871</td>\n",
       "      <td>0.1774</td>\n",
       "      <td>0.0097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>0.4721</td>\n",
       "      <td>0.4692</td>\n",
       "      <td>0.0029</td>\n",
       "      <td>0.2350</td>\n",
       "      <td>0.2141</td>\n",
       "      <td>0.0209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.4469</td>\n",
       "      <td>0.3660</td>\n",
       "      <td>0.0809</td>\n",
       "      <td>0.1680</td>\n",
       "      <td>0.0840</td>\n",
       "      <td>0.0840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>0.4426</td>\n",
       "      <td>0.4170</td>\n",
       "      <td>0.0256</td>\n",
       "      <td>0.2556</td>\n",
       "      <td>0.2340</td>\n",
       "      <td>0.0216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>0.4010</td>\n",
       "      <td>0.2041</td>\n",
       "      <td>0.1969</td>\n",
       "      <td>0.1744</td>\n",
       "      <td>0.1041</td>\n",
       "      <td>0.0703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>0.3475</td>\n",
       "      <td>0.3083</td>\n",
       "      <td>0.0392</td>\n",
       "      <td>0.2509</td>\n",
       "      <td>0.0501</td>\n",
       "      <td>0.2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>0.3307</td>\n",
       "      <td>0.3307</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2120</td>\n",
       "      <td>0.0831</td>\n",
       "      <td>0.1289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>0.3137</td>\n",
       "      <td>0.2994</td>\n",
       "      <td>0.0143</td>\n",
       "      <td>0.2136</td>\n",
       "      <td>0.1997</td>\n",
       "      <td>0.0139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>0.3111</td>\n",
       "      <td>0.2694</td>\n",
       "      <td>0.0417</td>\n",
       "      <td>0.1870</td>\n",
       "      <td>0.1314</td>\n",
       "      <td>0.0556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.2571</td>\n",
       "      <td>0.0406</td>\n",
       "      <td>0.2357</td>\n",
       "      <td>0.1388</td>\n",
       "      <td>0.0969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>0.2828</td>\n",
       "      <td>0.2828</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1461</td>\n",
       "      <td>0.1014</td>\n",
       "      <td>0.0447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>0.2728</td>\n",
       "      <td>0.2728</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1569</td>\n",
       "      <td>0.1486</td>\n",
       "      <td>0.0083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>0.2678</td>\n",
       "      <td>0.1672</td>\n",
       "      <td>0.1006</td>\n",
       "      <td>0.1797</td>\n",
       "      <td>0.1414</td>\n",
       "      <td>0.0383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>0.2536</td>\n",
       "      <td>0.2536</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0715</td>\n",
       "      <td>0.0492</td>\n",
       "      <td>0.0223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>0.2132</td>\n",
       "      <td>0.1639</td>\n",
       "      <td>0.0493</td>\n",
       "      <td>0.1168</td>\n",
       "      <td>0.0697</td>\n",
       "      <td>0.0471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>0.1851</td>\n",
       "      <td>0.1617</td>\n",
       "      <td>0.0234</td>\n",
       "      <td>0.0999</td>\n",
       "      <td>0.0645</td>\n",
       "      <td>0.0354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>0.1707</td>\n",
       "      <td>0.1707</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.1226</td>\n",
       "      <td>0.0977</td>\n",
       "      <td>0.0249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>0.1583</td>\n",
       "      <td>0.1489</td>\n",
       "      <td>0.0094</td>\n",
       "      <td>0.1095</td>\n",
       "      <td>0.0524</td>\n",
       "      <td>0.0571</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     test_accuracy_max  test_accuracy  test_diff  noise_accuracy_max  \\\n",
       "71              0.6731         0.6722     0.0009              0.2968   \n",
       "72              0.6720         0.6720     0.0000              0.2761   \n",
       "92              0.6707         0.6707     0.0000              0.2696   \n",
       "94              0.6706         0.6684     0.0022              0.2524   \n",
       "100             0.6683         0.6683     0.0000              0.2559   \n",
       "74              0.6669         0.6640     0.0029              0.2506   \n",
       "88              0.6667         0.6663     0.0004              0.2583   \n",
       "93              0.6604         0.6592     0.0012              0.2516   \n",
       "73              0.6566         0.6507     0.0059              0.2889   \n",
       "98              0.6545         0.6522     0.0023              0.2839   \n",
       "101             0.6475         0.6475     0.0000              0.2571   \n",
       "106             0.6461         0.6324     0.0137              0.2725   \n",
       "83              0.6428         0.6411     0.0017              0.2978   \n",
       "77              0.6284         0.6255     0.0029              0.2823   \n",
       "91              0.6124         0.6117     0.0007              0.2302   \n",
       "97              0.5928         0.5657     0.0271              0.2499   \n",
       "75              0.5913         0.5900     0.0013              0.2576   \n",
       "90              0.5574         0.5451     0.0123              0.3280   \n",
       "89              0.5424         0.5393     0.0031              0.1871   \n",
       "76              0.4721         0.4692     0.0029              0.2350   \n",
       "80              0.4469         0.3660     0.0809              0.1680   \n",
       "79              0.4426         0.4170     0.0256              0.2556   \n",
       "85              0.4010         0.2041     0.1969              0.1744   \n",
       "81              0.3475         0.3083     0.0392              0.2509   \n",
       "84              0.3307         0.3307     0.0000              0.2120   \n",
       "102             0.3137         0.2994     0.0143              0.2136   \n",
       "78              0.3111         0.2694     0.0417              0.1870   \n",
       "103             0.2977         0.2571     0.0406              0.2357   \n",
       "99              0.2828         0.2828     0.0000              0.1461   \n",
       "82              0.2728         0.2728     0.0000              0.1569   \n",
       "95              0.2678         0.1672     0.1006              0.1797   \n",
       "105             0.2536         0.2536     0.0000              0.0715   \n",
       "87              0.2132         0.1639     0.0493              0.1168   \n",
       "104             0.1851         0.1617     0.0234              0.0999   \n",
       "96              0.1707         0.1707     0.0000              0.1226   \n",
       "86              0.1583         0.1489     0.0094              0.1095   \n",
       "\n",
       "     noise_accuracy  noise_diff  \n",
       "71           0.2610      0.0358  \n",
       "72           0.2524      0.0237  \n",
       "92           0.2444      0.0252  \n",
       "94           0.2256      0.0268  \n",
       "100          0.2484      0.0075  \n",
       "74           0.2310      0.0196  \n",
       "88           0.2400      0.0183  \n",
       "93           0.2491      0.0025  \n",
       "73           0.2749      0.0140  \n",
       "98           0.2682      0.0157  \n",
       "101          0.2533      0.0038  \n",
       "106          0.2360      0.0365  \n",
       "83           0.2749      0.0229  \n",
       "77           0.2218      0.0605  \n",
       "91           0.2156      0.0146  \n",
       "97           0.2111      0.0388  \n",
       "75           0.2460      0.0116  \n",
       "90           0.3231      0.0049  \n",
       "89           0.1774      0.0097  \n",
       "76           0.2141      0.0209  \n",
       "80           0.0840      0.0840  \n",
       "79           0.2340      0.0216  \n",
       "85           0.1041      0.0703  \n",
       "81           0.0501      0.2008  \n",
       "84           0.0831      0.1289  \n",
       "102          0.1997      0.0139  \n",
       "78           0.1314      0.0556  \n",
       "103          0.1388      0.0969  \n",
       "99           0.1014      0.0447  \n",
       "82           0.1486      0.0083  \n",
       "95           0.1414      0.0383  \n",
       "105          0.0492      0.0223  \n",
       "87           0.0697      0.0471  \n",
       "104          0.0645      0.0354  \n",
       "96           0.0977      0.0249  \n",
       "86           0.0524      0.0571  "
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['noise_diff'] =  df['noise_accuracy_max'] - df['noise_accuracy']\n",
    "df['test_diff'] =  df['test_accuracy_max'] - df['test_accuracy']\n",
    "metrics = ['test_accuracy_max', 'test_accuracy', 'test_diff',\n",
    "           'noise_accuracy_max','noise_accuracy',  'noise_diff']\n",
    "\n",
    "(df[df['name'].str.startswith('C100_SparseSGD')][metrics]\n",
    "              .sort_values(['test_accuracy_max'], ascending=False))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_accuracy         0.356074\n",
       "test_accuracy_max     0.362631\n",
       "test_diff             0.006557\n",
       "noise_accuracy        0.164737\n",
       "noise_accuracy_max    0.190790\n",
       "noise_diff            0.026052\n",
       "dtype: float64"
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[metrics].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>epochs</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>early_stop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.6722</td>\n",
       "      <td>0.6731</td>\n",
       "      <td>0.2610</td>\n",
       "      <td>0.2968</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>102</td>\n",
       "      <td>0.6720</td>\n",
       "      <td>0.6720</td>\n",
       "      <td>0.2524</td>\n",
       "      <td>0.2761</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>58</td>\n",
       "      <td>0.6707</td>\n",
       "      <td>0.6707</td>\n",
       "      <td>0.2444</td>\n",
       "      <td>0.2696</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>52</td>\n",
       "      <td>0.6684</td>\n",
       "      <td>0.6706</td>\n",
       "      <td>0.2256</td>\n",
       "      <td>0.2524</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>48</td>\n",
       "      <td>0.6683</td>\n",
       "      <td>0.6683</td>\n",
       "      <td>0.2484</td>\n",
       "      <td>0.2559</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>98</td>\n",
       "      <td>0.6640</td>\n",
       "      <td>0.6669</td>\n",
       "      <td>0.2310</td>\n",
       "      <td>0.2506</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>62</td>\n",
       "      <td>0.6663</td>\n",
       "      <td>0.6667</td>\n",
       "      <td>0.2400</td>\n",
       "      <td>0.2583</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>50</td>\n",
       "      <td>0.6592</td>\n",
       "      <td>0.6604</td>\n",
       "      <td>0.2491</td>\n",
       "      <td>0.2516</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>102</td>\n",
       "      <td>0.6507</td>\n",
       "      <td>0.6566</td>\n",
       "      <td>0.2749</td>\n",
       "      <td>0.2889</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>55</td>\n",
       "      <td>0.6522</td>\n",
       "      <td>0.6545</td>\n",
       "      <td>0.2682</td>\n",
       "      <td>0.2839</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>49</td>\n",
       "      <td>0.6475</td>\n",
       "      <td>0.6475</td>\n",
       "      <td>0.2533</td>\n",
       "      <td>0.2571</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>60</td>\n",
       "      <td>0.6324</td>\n",
       "      <td>0.6461</td>\n",
       "      <td>0.2360</td>\n",
       "      <td>0.2725</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>54</td>\n",
       "      <td>0.6411</td>\n",
       "      <td>0.6428</td>\n",
       "      <td>0.2749</td>\n",
       "      <td>0.2978</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>42</td>\n",
       "      <td>0.6255</td>\n",
       "      <td>0.6284</td>\n",
       "      <td>0.2218</td>\n",
       "      <td>0.2823</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>44</td>\n",
       "      <td>0.6117</td>\n",
       "      <td>0.6124</td>\n",
       "      <td>0.2156</td>\n",
       "      <td>0.2302</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>60</td>\n",
       "      <td>0.5657</td>\n",
       "      <td>0.5928</td>\n",
       "      <td>0.2111</td>\n",
       "      <td>0.2499</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>84</td>\n",
       "      <td>0.5900</td>\n",
       "      <td>0.5913</td>\n",
       "      <td>0.2460</td>\n",
       "      <td>0.2576</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>60</td>\n",
       "      <td>0.5451</td>\n",
       "      <td>0.5574</td>\n",
       "      <td>0.3231</td>\n",
       "      <td>0.3280</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>54</td>\n",
       "      <td>0.5393</td>\n",
       "      <td>0.5424</td>\n",
       "      <td>0.1774</td>\n",
       "      <td>0.1871</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>81</td>\n",
       "      <td>0.4692</td>\n",
       "      <td>0.4721</td>\n",
       "      <td>0.2141</td>\n",
       "      <td>0.2350</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.3660</td>\n",
       "      <td>0.4469</td>\n",
       "      <td>0.0840</td>\n",
       "      <td>0.1680</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.4170</td>\n",
       "      <td>0.4426</td>\n",
       "      <td>0.2340</td>\n",
       "      <td>0.2556</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2041</td>\n",
       "      <td>0.4010</td>\n",
       "      <td>0.1041</td>\n",
       "      <td>0.1744</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.3083</td>\n",
       "      <td>0.3475</td>\n",
       "      <td>0.0501</td>\n",
       "      <td>0.2509</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.3307</td>\n",
       "      <td>0.3307</td>\n",
       "      <td>0.0831</td>\n",
       "      <td>0.2120</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2994</td>\n",
       "      <td>0.3137</td>\n",
       "      <td>0.1997</td>\n",
       "      <td>0.2136</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2694</td>\n",
       "      <td>0.3111</td>\n",
       "      <td>0.1314</td>\n",
       "      <td>0.1870</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2571</td>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.1388</td>\n",
       "      <td>0.2357</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2828</td>\n",
       "      <td>0.2828</td>\n",
       "      <td>0.1014</td>\n",
       "      <td>0.1461</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2728</td>\n",
       "      <td>0.2728</td>\n",
       "      <td>0.1486</td>\n",
       "      <td>0.1569</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.1672</td>\n",
       "      <td>0.2678</td>\n",
       "      <td>0.1414</td>\n",
       "      <td>0.1797</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.2536</td>\n",
       "      <td>0.2536</td>\n",
       "      <td>0.0492</td>\n",
       "      <td>0.0715</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.1639</td>\n",
       "      <td>0.2132</td>\n",
       "      <td>0.0697</td>\n",
       "      <td>0.1168</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.1617</td>\n",
       "      <td>0.1851</td>\n",
       "      <td>0.0645</td>\n",
       "      <td>0.0999</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.1707</td>\n",
       "      <td>0.1707</td>\n",
       "      <td>0.0977</td>\n",
       "      <td>0.1226</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>C100_SparseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.1489</td>\n",
       "      <td>0.1583</td>\n",
       "      <td>0.0524</td>\n",
       "      <td>0.1095</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               name  epochs  test_accuracy  test_accuracy_max  noise_accuracy  \\\n",
       "71   C100_SparseSGD     164         0.6722             0.6731          0.2610   \n",
       "72   C100_SparseSGD     102         0.6720             0.6720          0.2524   \n",
       "92   C100_SparseSGD      58         0.6707             0.6707          0.2444   \n",
       "94   C100_SparseSGD      52         0.6684             0.6706          0.2256   \n",
       "100  C100_SparseSGD      48         0.6683             0.6683          0.2484   \n",
       "74   C100_SparseSGD      98         0.6640             0.6669          0.2310   \n",
       "88   C100_SparseSGD      62         0.6663             0.6667          0.2400   \n",
       "93   C100_SparseSGD      50         0.6592             0.6604          0.2491   \n",
       "73   C100_SparseSGD     102         0.6507             0.6566          0.2749   \n",
       "98   C100_SparseSGD      55         0.6522             0.6545          0.2682   \n",
       "101  C100_SparseSGD      49         0.6475             0.6475          0.2533   \n",
       "106  C100_SparseSGD      60         0.6324             0.6461          0.2360   \n",
       "83   C100_SparseSGD      54         0.6411             0.6428          0.2749   \n",
       "77   C100_SparseSGD      42         0.6255             0.6284          0.2218   \n",
       "91   C100_SparseSGD      44         0.6117             0.6124          0.2156   \n",
       "97   C100_SparseSGD      60         0.5657             0.5928          0.2111   \n",
       "75   C100_SparseSGD      84         0.5900             0.5913          0.2460   \n",
       "90   C100_SparseSGD      60         0.5451             0.5574          0.3231   \n",
       "89   C100_SparseSGD      54         0.5393             0.5424          0.1774   \n",
       "76   C100_SparseSGD      81         0.4692             0.4721          0.2141   \n",
       "80   C100_SparseSGD      20         0.3660             0.4469          0.0840   \n",
       "79   C100_SparseSGD      20         0.4170             0.4426          0.2340   \n",
       "85   C100_SparseSGD      20         0.2041             0.4010          0.1041   \n",
       "81   C100_SparseSGD      20         0.3083             0.3475          0.0501   \n",
       "84   C100_SparseSGD      20         0.3307             0.3307          0.0831   \n",
       "102  C100_SparseSGD      20         0.2994             0.3137          0.1997   \n",
       "78   C100_SparseSGD      20         0.2694             0.3111          0.1314   \n",
       "103  C100_SparseSGD      20         0.2571             0.2977          0.1388   \n",
       "99   C100_SparseSGD      20         0.2828             0.2828          0.1014   \n",
       "82   C100_SparseSGD      20         0.2728             0.2728          0.1486   \n",
       "95   C100_SparseSGD      20         0.1672             0.2678          0.1414   \n",
       "105  C100_SparseSGD      20         0.2536             0.2536          0.0492   \n",
       "87   C100_SparseSGD      20         0.1639             0.2132          0.0697   \n",
       "104  C100_SparseSGD      20         0.1617             0.1851          0.0645   \n",
       "96   C100_SparseSGD      20         0.1707             0.1707          0.0977   \n",
       "86   C100_SparseSGD      20         0.1489             0.1583          0.0524   \n",
       "\n",
       "     noise_accuracy_max  early_stop  \n",
       "71               0.2968           0  \n",
       "72               0.2761           1  \n",
       "92               0.2696           1  \n",
       "94               0.2524           1  \n",
       "100              0.2559           1  \n",
       "74               0.2506           1  \n",
       "88               0.2583           1  \n",
       "93               0.2516           1  \n",
       "73               0.2889           1  \n",
       "98               0.2839           1  \n",
       "101              0.2571           1  \n",
       "106              0.2725           0  \n",
       "83               0.2978           1  \n",
       "77               0.2823           1  \n",
       "91               0.2302           1  \n",
       "97               0.2499           0  \n",
       "75               0.2576           1  \n",
       "90               0.3280           0  \n",
       "89               0.1871           1  \n",
       "76               0.2350           1  \n",
       "80               0.1680           0  \n",
       "79               0.2556           0  \n",
       "85               0.1744           0  \n",
       "81               0.2509           0  \n",
       "84               0.2120           0  \n",
       "102              0.2136           0  \n",
       "78               0.1870           0  \n",
       "103              0.2357           0  \n",
       "99               0.1461           0  \n",
       "82               0.1569           0  \n",
       "95               0.1797           0  \n",
       "105              0.0715           0  \n",
       "87               0.1168           0  \n",
       "104              0.0999           0  \n",
       "96               0.1226           0  \n",
       "86               0.1095           0  "
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = ['epochs', 'test_accuracy', 'test_accuracy_max', 'noise_accuracy', 'noise_accuracy_max', 'early_stop']\n",
    "df[df['name'].str.startswith('C100_SparseSGD')][['name'] + metrics].sort_values(['test_accuracy_max'], ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>epochs</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>early_stop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>51</td>\n",
       "      <td>0.7115</td>\n",
       "      <td>0.7124</td>\n",
       "      <td>0.2102</td>\n",
       "      <td>0.2676</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>42</td>\n",
       "      <td>0.7124</td>\n",
       "      <td>0.7124</td>\n",
       "      <td>0.2231</td>\n",
       "      <td>0.2433</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>42</td>\n",
       "      <td>0.7055</td>\n",
       "      <td>0.7057</td>\n",
       "      <td>0.1957</td>\n",
       "      <td>0.2495</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>91</td>\n",
       "      <td>0.7007</td>\n",
       "      <td>0.7024</td>\n",
       "      <td>0.2234</td>\n",
       "      <td>0.2434</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>43</td>\n",
       "      <td>0.6990</td>\n",
       "      <td>0.6996</td>\n",
       "      <td>0.2086</td>\n",
       "      <td>0.2336</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>42</td>\n",
       "      <td>0.6936</td>\n",
       "      <td>0.6980</td>\n",
       "      <td>0.1976</td>\n",
       "      <td>0.2203</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>42</td>\n",
       "      <td>0.6928</td>\n",
       "      <td>0.6946</td>\n",
       "      <td>0.2310</td>\n",
       "      <td>0.2594</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.6836</td>\n",
       "      <td>0.6875</td>\n",
       "      <td>0.2044</td>\n",
       "      <td>0.2199</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>95</td>\n",
       "      <td>0.6772</td>\n",
       "      <td>0.6798</td>\n",
       "      <td>0.2124</td>\n",
       "      <td>0.2339</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.6746</td>\n",
       "      <td>0.6766</td>\n",
       "      <td>0.2211</td>\n",
       "      <td>0.2313</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>89</td>\n",
       "      <td>0.6697</td>\n",
       "      <td>0.6759</td>\n",
       "      <td>0.2105</td>\n",
       "      <td>0.2442</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>110</td>\n",
       "      <td>0.6559</td>\n",
       "      <td>0.6580</td>\n",
       "      <td>0.2089</td>\n",
       "      <td>0.2188</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>164</td>\n",
       "      <td>0.6507</td>\n",
       "      <td>0.6529</td>\n",
       "      <td>0.2011</td>\n",
       "      <td>0.2125</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.5566</td>\n",
       "      <td>0.5881</td>\n",
       "      <td>0.1333</td>\n",
       "      <td>0.1803</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.5627</td>\n",
       "      <td>0.5627</td>\n",
       "      <td>0.1055</td>\n",
       "      <td>0.2030</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.5035</td>\n",
       "      <td>0.5445</td>\n",
       "      <td>0.1713</td>\n",
       "      <td>0.2223</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>C100_DenseSGD</td>\n",
       "      <td>20</td>\n",
       "      <td>0.5294</td>\n",
       "      <td>0.5309</td>\n",
       "      <td>0.1735</td>\n",
       "      <td>0.2141</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             name  epochs  test_accuracy  test_accuracy_max  noise_accuracy  \\\n",
       "28  C100_DenseSGD      51         0.7115             0.7124          0.2102   \n",
       "31  C100_DenseSGD      42         0.7124             0.7124          0.2231   \n",
       "26  C100_DenseSGD      42         0.7055             0.7057          0.1957   \n",
       "23  C100_DenseSGD      91         0.7007             0.7024          0.2234   \n",
       "27  C100_DenseSGD      43         0.6990             0.6996          0.2086   \n",
       "32  C100_DenseSGD      42         0.6936             0.6980          0.1976   \n",
       "30  C100_DenseSGD      42         0.6928             0.6946          0.2310   \n",
       "19  C100_DenseSGD     164         0.6836             0.6875          0.2044   \n",
       "22  C100_DenseSGD      95         0.6772             0.6798          0.2124   \n",
       "20  C100_DenseSGD     164         0.6746             0.6766          0.2211   \n",
       "24  C100_DenseSGD      89         0.6697             0.6759          0.2105   \n",
       "21  C100_DenseSGD     110         0.6559             0.6580          0.2089   \n",
       "18  C100_DenseSGD     164         0.6507             0.6529          0.2011   \n",
       "25  C100_DenseSGD      20         0.5566             0.5881          0.1333   \n",
       "34  C100_DenseSGD      20         0.5627             0.5627          0.1055   \n",
       "29  C100_DenseSGD      20         0.5035             0.5445          0.1713   \n",
       "33  C100_DenseSGD      20         0.5294             0.5309          0.1735   \n",
       "\n",
       "    noise_accuracy_max  early_stop  \n",
       "28              0.2676           1  \n",
       "31              0.2433           1  \n",
       "26              0.2495           1  \n",
       "23              0.2434           1  \n",
       "27              0.2336           1  \n",
       "32              0.2203           1  \n",
       "30              0.2594           1  \n",
       "19              0.2199           0  \n",
       "22              0.2339           1  \n",
       "20              0.2313           0  \n",
       "24              0.2442           1  \n",
       "21              0.2188           1  \n",
       "18              0.2125           0  \n",
       "25              0.1803           0  \n",
       "34              0.2030           0  \n",
       "29              0.2223           0  \n",
       "33              0.2141           0  "
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = ['epochs', 'test_accuracy', 'test_accuracy_max', 'noise_accuracy', 'noise_accuracy_max', 'early_stop']\n",
    "df[df['name'].str.startswith('C100_DenseSGD')][['name'] + metrics].sort_values(['test_accuracy_max'], ascending=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Too many early stops, even in promising cases (analysis also done on Tensorboard plots). The main issue seems to be that early stopping is based on the mean accuracy right now. At some point during training, the test training continues to increase while the noise accuracy starts to oscillate, many times decreasing a little bit, causing the mean accuracy to plateau and early stop signal to fire. \n",
    "- Best way to fix would be to set a signal that looks both at the test and the noise accuracy independently, and would only fire if both of them plateau. That would ensure we could drive promising experiments on the noise accuracy metric and the test accuracy metric all the way to the end, to get the best possible results."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What are the best possible parameters?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Can answer this question regarding:\n",
    "- test accuracy\n",
    "- mean accuracy\n",
    "- noise accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "tunable_params_general = ['learning_rate', 'learning_rate_gamma', 'weight_decay', \n",
    "                          'momentum', 'batch_size', 'batches_in_epoch']\n",
    "tunable_params_sparsity = ['boost_strength', 'boost_strength_factor', \n",
    "                           'k_inference_factor', 'cnn_percent_on', 'cnn_weight_sparsity']\n",
    "tunable_params = tunable_params_general + tunable_params_sparsity\n",
    "performance_metrics = ['test_accuracy_max', 'mean_accuracy_max', 'noise_accuracy_max']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse_exps = [\n",
    "  'C100_SparseAdam',\n",
    "  'C100_SparseSGD',\n",
    "  'C10_SparseAdam',\n",
    "  'C10_SparseSGD',\n",
    "  'VGG19SparseFull',\n",
    "  'VGG19SparseFull-short',\n",
    "  'VGG19SparseTest9b2',\n",
    "]\n",
    "\n",
    "dense_exps = [\n",
    "  'VGG19DenseTest9v2',\n",
    "  'C100_DenseAdam',\n",
    "  'C100_DenseSGD',\n",
    "  'C10_DenseAdam',\n",
    "  'C10_DenseSGD',  \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "performance_metrics = ['test_accuracy_max', 'mean_accuracy_max', 'noise_accuracy_max']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stats(arr):\n",
    "  mean = np.mean(arr)\n",
    "  std = np.std(arr)\n",
    "  return [round(v, 4) for v in [mean-std, mean, mean+std]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate            [0.0696, 0.0817, 0.0939]\n",
       "learning_rate_gamma      [0.2215, 0.3903, 0.5592]\n",
       "weight_decay              [0.0024, 0.004, 0.0057]\n",
       "momentum                 [0.4172, 0.4472, 0.4772]\n",
       "batch_size                  [128.0, 128.0, 128.0]\n",
       "batches_in_epoch            [500.0, 500.0, 500.0]\n",
       "boost_strength            [1.5338, 1.698, 1.8622]\n",
       "boost_strength_factor     [0.6667, 0.743, 0.8192]\n",
       "k_inference_factor       [0.9434, 1.0053, 1.0672]\n",
       "cnn_percent_on           [0.2734, 0.2969, 0.3204]\n",
       "cnn_weight_sparsity      [0.5521, 0.6233, 0.6945]\n",
       "dtype: object"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-10 SPARSE\n",
    "filters = ((df['dataset']=='CIFAR10') & \n",
    "           (df['name'].isin(sparse_exps)) &\n",
    "           (df['optimizer'] != 'Adam'))\n",
    "         \n",
    "(df[filters]\n",
    "  .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "  .head(5)\n",
    "  .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate            [0.0696, 0.0817, 0.0939]\n",
       "learning_rate_gamma      [0.2215, 0.3903, 0.5592]\n",
       "weight_decay              [0.0024, 0.004, 0.0057]\n",
       "momentum                 [0.4172, 0.4472, 0.4772]\n",
       "batch_size                  [128.0, 128.0, 128.0]\n",
       "batches_in_epoch            [500.0, 500.0, 500.0]\n",
       "boost_strength            [1.5338, 1.698, 1.8622]\n",
       "boost_strength_factor     [0.6667, 0.743, 0.8192]\n",
       "k_inference_factor       [0.9434, 1.0053, 1.0672]\n",
       "cnn_percent_on           [0.2734, 0.2969, 0.3204]\n",
       "cnn_weight_sparsity      [0.5521, 0.6233, 0.6945]\n",
       "dtype: object"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-10 SPARSE\n",
    "filters = ((df['dataset']=='CIFAR10') & \n",
    "           (df['name'].isin(sparse_exps)) &\n",
    "           (df['optimizer'] != 'Adam'))\n",
    "         \n",
    "(df[filters]\n",
    "  .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "  .head(5)\n",
    "  .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate               [0.0888, 0.1019, 0.1151]\n",
       "learning_rate_gamma          [0.0613, 0.121, 0.1807]\n",
       "weight_decay                [0.0004, 0.0006, 0.0008]\n",
       "momentum                    [0.2698, 0.4052, 0.5407]\n",
       "batch_size                [71.0465, 102.4, 133.7535]\n",
       "batches_in_epoch         [377.5694, 454.8, 532.0306]\n",
       "boost_strength              [1.1203, 1.4171, 1.7139]\n",
       "boost_strength_factor       [0.6079, 0.7184, 0.8288]\n",
       "k_inference_factor          [0.9137, 0.9898, 1.0659]\n",
       "cnn_percent_on              [0.3158, 0.3285, 0.3412]\n",
       "cnn_weight_sparsity         [0.8039, 0.8485, 0.8931]\n",
       "dtype: object"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-100 SPARSE\n",
    "filters = ((df['dataset']=='CIFAR100') & \n",
    "           (df['name'].isin(sparse_exps)) &\n",
    "           (df['optimizer'] != 'Adam'))\n",
    "         \n",
    "(df[filters]\n",
    "  .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "  .head(5)\n",
    "  .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate               [0.0888, 0.1019, 0.1151]\n",
       "learning_rate_gamma          [0.0613, 0.121, 0.1807]\n",
       "weight_decay                [0.0004, 0.0006, 0.0008]\n",
       "momentum                    [0.2698, 0.4052, 0.5407]\n",
       "batch_size                [71.0465, 102.4, 133.7535]\n",
       "batches_in_epoch         [377.5694, 454.8, 532.0306]\n",
       "boost_strength              [1.1203, 1.4171, 1.7139]\n",
       "boost_strength_factor       [0.6079, 0.7184, 0.8288]\n",
       "k_inference_factor          [0.9137, 0.9898, 1.0659]\n",
       "cnn_percent_on              [0.3158, 0.3285, 0.3412]\n",
       "cnn_weight_sparsity         [0.8039, 0.8485, 0.8931]\n",
       "dtype: object"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-100 SPARSE\n",
    "filters = ((df['dataset']=='CIFAR100') & \n",
    "           (df['name'].isin(sparse_exps)) &\n",
    "           (df['optimizer'] != 'Adam'))\n",
    "         \n",
    "(df[filters]\n",
    "  .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "  .head(5)\n",
    "  .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate               [0.0724, 0.0855, 0.0986]\n",
       "learning_rate_gamma         [0.1115, 0.1926, 0.2738]\n",
       "weight_decay                [0.0003, 0.0024, 0.0044]\n",
       "momentum                     [0.488, 0.5291, 0.5703]\n",
       "batch_size                     [128.0, 128.0, 128.0]\n",
       "batches_in_epoch         [411.0102, 460.0, 508.9898]\n",
       "boost_strength                       [1.5, 1.5, 1.5]\n",
       "boost_strength_factor             [0.85, 0.85, 0.85]\n",
       "k_inference_factor                   [1.0, 1.0, 1.0]\n",
       "cnn_percent_on                       [1.0, 1.0, 1.0]\n",
       "cnn_weight_sparsity                  [1.0, 1.0, 1.0]\n",
       "dtype: object"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-100 DENSE\n",
    "filters = ((df['dataset']=='CIFAR100') & \n",
    "           (df['name'].isin(dense_exps)) &\n",
    "           (df['optimizer'] != 'Adam'))\n",
    "         \n",
    "(df[filters]\n",
    "  .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "  .head(5)\n",
    "  .apply(stats))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Best results on original experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate                [0.066, 0.0884, 0.1108]\n",
       "learning_rate_gamma          [0.063, 0.1146, 0.1662]\n",
       "weight_decay                [0.0004, 0.0007, 0.0009]\n",
       "momentum                    [0.3304, 0.4923, 0.6542]\n",
       "batch_size                [79.4715, 108.8, 138.1285]\n",
       "batches_in_epoch         [389.4726, 469.0, 548.5274]\n",
       "boost_strength              [1.1116, 1.4042, 1.6968]\n",
       "boost_strength_factor       [0.5602, 0.6985, 0.8369]\n",
       "k_inference_factor          [0.8969, 0.9903, 1.0838]\n",
       "cnn_percent_on              [0.2813, 0.3133, 0.3452]\n",
       "cnn_weight_sparsity           [0.8023, 0.86, 0.9177]\n",
       "dtype: object"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-100 SPARSE - VGG19SparseFull\n",
    "filters = ((df['dataset']=='CIFAR100') & \n",
    "           (df['optimizer'] != 'Adam') &\n",
    "           (df['name'] == 'VGG19SparseFull'))\n",
    "         \n",
    "(df[filters]\n",
    "  .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "  .head(10)\n",
    "  .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate               [0.0865, 0.1061, 0.1257]\n",
       "learning_rate_gamma         [0.0663, 0.1031, 0.1399]\n",
       "weight_decay                [0.0007, 0.0008, 0.0009]\n",
       "momentum                     [0.3612, 0.541, 0.7209]\n",
       "batch_size                [79.4715, 108.8, 138.1285]\n",
       "batches_in_epoch         [399.0448, 478.4, 557.7552]\n",
       "boost_strength              [1.1288, 1.4225, 1.7163]\n",
       "boost_strength_factor        [0.534, 0.6551, 0.7762]\n",
       "k_inference_factor           [0.9117, 0.998, 1.0842]\n",
       "cnn_percent_on              [0.2672, 0.3028, 0.3385]\n",
       "cnn_weight_sparsity          [0.6838, 0.817, 0.9501]\n",
       "dtype: object"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-100 SPARSE - VGG19SparseFull\n",
    "filters = ((df['dataset']=='CIFAR100') & \n",
    "           (df['optimizer'] != 'Adam') &\n",
    "           (df['name'] == 'VGG19SparseFull'))\n",
    "         \n",
    "(df[filters]\n",
    "  .sort_values('test_accuracy_max', ascending=False)[tunable_params]\n",
    "  .head(10)\n",
    "  .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate               [0.0521, 0.0688, 0.0855]\n",
       "learning_rate_gamma         [0.0771, 0.1243, 0.1714]\n",
       "weight_decay                [0.0004, 0.0007, 0.0009]\n",
       "momentum                    [0.3749, 0.5167, 0.6585]\n",
       "batch_size                [71.0465, 102.4, 133.7535]\n",
       "batches_in_epoch         [381.7528, 469.4, 557.0472]\n",
       "boost_strength              [1.1072, 1.4464, 1.7856]\n",
       "boost_strength_factor       [0.5765, 0.7248, 0.8731]\n",
       "k_inference_factor          [0.9118, 0.9959, 1.0801]\n",
       "cnn_percent_on              [0.2197, 0.2663, 0.3128]\n",
       "cnn_weight_sparsity          [0.5639, 0.777, 0.9901]\n",
       "dtype: object"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-100 SPARSE - VGG19SparseFull\n",
    "filters = ((df['dataset']=='CIFAR100') & \n",
    "           (df['optimizer'] != 'Adam') &\n",
    "           (df['name'] == 'VGG19SparseFull'))\n",
    "         \n",
    "(df[filters]\n",
    "  .sort_values('noise_accuracy_max', ascending=False)[tunable_params]\n",
    "  .head(10)\n",
    "  .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epochs</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>164</td>\n",
       "      <td>0.48625</td>\n",
       "      <td>0.6923</td>\n",
       "      <td>0.2808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417</th>\n",
       "      <td>164</td>\n",
       "      <td>0.48490</td>\n",
       "      <td>0.6660</td>\n",
       "      <td>0.3203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>164</td>\n",
       "      <td>0.48375</td>\n",
       "      <td>0.6757</td>\n",
       "      <td>0.3014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>164</td>\n",
       "      <td>0.48325</td>\n",
       "      <td>0.6955</td>\n",
       "      <td>0.3005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>164</td>\n",
       "      <td>0.48260</td>\n",
       "      <td>0.6984</td>\n",
       "      <td>0.2762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>164</td>\n",
       "      <td>0.48010</td>\n",
       "      <td>0.6597</td>\n",
       "      <td>0.3216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>90</td>\n",
       "      <td>0.47985</td>\n",
       "      <td>0.7005</td>\n",
       "      <td>0.2821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>164</td>\n",
       "      <td>0.47960</td>\n",
       "      <td>0.6822</td>\n",
       "      <td>0.3143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>164</td>\n",
       "      <td>0.47875</td>\n",
       "      <td>0.6954</td>\n",
       "      <td>0.2872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>90</td>\n",
       "      <td>0.47870</td>\n",
       "      <td>0.6990</td>\n",
       "      <td>0.2932</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     epochs  mean_accuracy_max  test_accuracy_max  noise_accuracy_max\n",
       "259     164            0.48625             0.6923              0.2808\n",
       "417     164            0.48490             0.6660              0.3203\n",
       "403     164            0.48375             0.6757              0.3014\n",
       "440     164            0.48325             0.6955              0.3005\n",
       "456     164            0.48260             0.6984              0.2762\n",
       "292     164            0.48010             0.6597              0.3216\n",
       "349      90            0.47985             0.7005              0.2821\n",
       "245     164            0.47960             0.6822              0.3143\n",
       "307     164            0.47875             0.6954              0.2872\n",
       "443      90            0.47870             0.6990              0.2932"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = ['epochs', 'mean_accuracy_max', 'test_accuracy_max', 'noise_accuracy_max']\n",
    "(df[df['name'].str.startswith('VGG19SparseFull')][metrics]\n",
    "              .sort_values(['mean_accuracy_max'], ascending=False)\n",
    "              .iloc[:10])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epochs</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>42</td>\n",
       "      <td>0.47185</td>\n",
       "      <td>0.6946</td>\n",
       "      <td>0.2594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>51</td>\n",
       "      <td>0.47050</td>\n",
       "      <td>0.7124</td>\n",
       "      <td>0.2676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>42</td>\n",
       "      <td>0.46985</td>\n",
       "      <td>0.7124</td>\n",
       "      <td>0.2433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>91</td>\n",
       "      <td>0.46515</td>\n",
       "      <td>0.7024</td>\n",
       "      <td>0.2434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>42</td>\n",
       "      <td>0.46320</td>\n",
       "      <td>0.7057</td>\n",
       "      <td>0.2495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>43</td>\n",
       "      <td>0.46165</td>\n",
       "      <td>0.6996</td>\n",
       "      <td>0.2336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>42</td>\n",
       "      <td>0.45705</td>\n",
       "      <td>0.6980</td>\n",
       "      <td>0.2203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>164</td>\n",
       "      <td>0.44995</td>\n",
       "      <td>0.6766</td>\n",
       "      <td>0.2313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>89</td>\n",
       "      <td>0.44970</td>\n",
       "      <td>0.6759</td>\n",
       "      <td>0.2442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>95</td>\n",
       "      <td>0.44805</td>\n",
       "      <td>0.6798</td>\n",
       "      <td>0.2339</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    epochs  mean_accuracy_max  test_accuracy_max  noise_accuracy_max\n",
       "30      42            0.47185             0.6946              0.2594\n",
       "28      51            0.47050             0.7124              0.2676\n",
       "31      42            0.46985             0.7124              0.2433\n",
       "23      91            0.46515             0.7024              0.2434\n",
       "26      42            0.46320             0.7057              0.2495\n",
       "27      43            0.46165             0.6996              0.2336\n",
       "32      42            0.45705             0.6980              0.2203\n",
       "20     164            0.44995             0.6766              0.2313\n",
       "24      89            0.44970             0.6759              0.2442\n",
       "22      95            0.44805             0.6798              0.2339"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics = ['epochs', 'mean_accuracy_max', 'test_accuracy_max', 'noise_accuracy_max']\n",
    "(df[df['name'].str.startswith('C100_DenseSGD')][metrics]\n",
    "              .sort_values(['mean_accuracy_max'], ascending=False)\n",
    "              .iloc[:10])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning Rate Decay Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_learning_rate</th>\n",
       "      <th>end_learning_rate</th>\n",
       "      <th>early_stop</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>epochs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>0.0010</td>\n",
       "      <td>1.000000e-03</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3723</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>0.0100</td>\n",
       "      <td>1.000000e-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2326</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0.0030</td>\n",
       "      <td>3.000000e-03</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1720</td>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>1.000000e-04</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6136</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>0.0030</td>\n",
       "      <td>3.000000e-03</td>\n",
       "      <td>1</td>\n",
       "      <td>0.4469</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>1.000000e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0546</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>4.733613e-05</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5884</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>0.0300</td>\n",
       "      <td>1.440304e-03</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0384</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>3.342565e-06</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>4.296152e-05</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6153</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>0.0010</td>\n",
       "      <td>1.000000e-03</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2946</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>0.0010</td>\n",
       "      <td>1.000000e-03</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3830</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>1.114699e-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0247</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>3.310691e-06</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6552</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.0030</td>\n",
       "      <td>2.121552e-03</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1680</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>3.505160e-07</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5390</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>1.286253e-03</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>8.182201e-05</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5712</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>1.645728e-04</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5942</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>0.1000</td>\n",
       "      <td>1.567386e-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    start_learning_rate  end_learning_rate  early_stop  test_accuracy  epochs\n",
       "35               0.0010       1.000000e-03           0         0.3723     164\n",
       "36               0.0100       1.000000e-02           0         0.2326     164\n",
       "37               0.0030       3.000000e-03           1         0.1720      99\n",
       "38               0.0001       1.000000e-04           1         0.6136      96\n",
       "39               0.0030       3.000000e-03           1         0.4469      96\n",
       "40               0.1000       1.000000e-01           1         0.0546      82\n",
       "41               0.0003       4.733613e-05           1         0.5884      49\n",
       "42               0.0300       1.440304e-03           0         0.0384      20\n",
       "43               0.1000       3.342565e-06           1         0.0100     192\n",
       "44               0.0003       4.296152e-05           1         0.6153      49\n",
       "45               0.0010       1.000000e-03           0         0.2946      20\n",
       "46               0.0010       1.000000e-03           0         0.3830      20\n",
       "47               0.1000       1.114699e-02           0         0.0247      20\n",
       "48               0.0001       3.310691e-06           1         0.6552      51\n",
       "49               0.0030       2.121552e-03           0         0.1680      20\n",
       "50               0.0003       3.505160e-07           1         0.5390      48\n",
       "51               0.1000       1.286253e-03           0         0.0100      20\n",
       "52               0.0003       8.182201e-05           1         0.5712      57\n",
       "53               0.0003       1.645728e-04           1         0.5942      51\n",
       "54               0.1000       1.567386e-02           0         0.0100      20"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filters = (df['name'].str.startswith('C100_SparseAdam'))\n",
    "         \n",
    "(df[filters]\n",
    "  [['start_learning_rate', 'end_learning_rate', 'early_stop', 'test_accuracy', 'epochs']]\n",
    "  .head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_learning_rate</th>\n",
       "      <th>end_learning_rate</th>\n",
       "      <th>early_stop</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>epochs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>0.062032</td>\n",
       "      <td>0.062032</td>\n",
       "      <td>0</td>\n",
       "      <td>0.6722</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>0.073143</td>\n",
       "      <td>0.073143</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6720</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>0.089823</td>\n",
       "      <td>0.089823</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6507</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>0.094103</td>\n",
       "      <td>0.094103</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6640</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>0.089010</td>\n",
       "      <td>0.089010</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5900</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>0.084460</td>\n",
       "      <td>0.084460</td>\n",
       "      <td>1</td>\n",
       "      <td>0.4692</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>0.077336</td>\n",
       "      <td>0.001499</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6255</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>0.083494</td>\n",
       "      <td>0.062876</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2694</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>0.097652</td>\n",
       "      <td>0.024427</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4170</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.074639</td>\n",
       "      <td>0.036643</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3660</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>0.101649</td>\n",
       "      <td>0.101649</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3083</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>0.101533</td>\n",
       "      <td>0.101533</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2728</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>0.071089</td>\n",
       "      <td>0.000323</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6411</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>0.089973</td>\n",
       "      <td>0.051376</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3307</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>0.099974</td>\n",
       "      <td>0.051785</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2041</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>0.103079</td>\n",
       "      <td>0.103079</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1489</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>0.115307</td>\n",
       "      <td>0.077382</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1639</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>0.079804</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6663</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>0.096964</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5393</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.071324</td>\n",
       "      <td>0.009044</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5451</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    start_learning_rate  end_learning_rate  early_stop  test_accuracy  epochs\n",
       "71             0.062032           0.062032           0         0.6722     164\n",
       "72             0.073143           0.073143           1         0.6720     102\n",
       "73             0.089823           0.089823           1         0.6507     102\n",
       "74             0.094103           0.094103           1         0.6640      98\n",
       "75             0.089010           0.089010           1         0.5900      84\n",
       "76             0.084460           0.084460           1         0.4692      81\n",
       "77             0.077336           0.001499           1         0.6255      42\n",
       "78             0.083494           0.062876           0         0.2694      20\n",
       "79             0.097652           0.024427           0         0.4170      20\n",
       "80             0.074639           0.036643           0         0.3660      20\n",
       "81             0.101649           0.101649           0         0.3083      20\n",
       "82             0.101533           0.101533           0         0.2728      20\n",
       "83             0.071089           0.000323           1         0.6411      54\n",
       "84             0.089973           0.051376           0         0.3307      20\n",
       "85             0.099974           0.051785           0         0.2041      20\n",
       "86             0.103079           0.103079           0         0.1489      20\n",
       "87             0.115307           0.077382           0         0.1639      20\n",
       "88             0.079804           0.000004           1         0.6663      62\n",
       "89             0.096964           0.000001           1         0.5393      54\n",
       "90             0.071324           0.009044           0         0.5451      60"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filters = (df['name'].str.startswith('C100_SparseSGD'))\n",
    "         \n",
    "(df[filters]\n",
    "  [['start_learning_rate', 'end_learning_rate', 'early_stop', 'test_accuracy', 'epochs']]\n",
    "  .head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
